{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from utils import *\n",
    "from constant import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>win_ratio</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>recent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>~speedup1-yz0-~opentop1-top1-3</th>\n",
       "      <td>0.618557</td>\n",
       "      <td>1.020334</td>\n",
       "      <td>97</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jump1-yz0-~opentop1-top1-3</th>\n",
       "      <td>0.632184</td>\n",
       "      <td>1.021891</td>\n",
       "      <td>87</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jump2-yz0-~opentop1-top1-3</th>\n",
       "      <td>0.717949</td>\n",
       "      <td>1.038423</td>\n",
       "      <td>39</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                win_ratio      mean  count recent\n",
       "~speedup1-yz0-~opentop1-top1-3   0.618557  1.020334     97      3\n",
       "jump1-yz0-~opentop1-top1-3       0.632184  1.021891     87      3\n",
       "jump2-yz0-~opentop1-top1-3       0.717949  1.038423     39      3"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"output3.csv\",index_col=0)\n",
    "df = df[~df[\"mean\"].duplicated()]\n",
    "df[\"recent\"] = df.index.str.slice(-1)\n",
    "df = df[df[\"count\"] > 20]\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9421"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'remove_invalid' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-1aa0a27c82a4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mvalid_keys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'~opentop2-jump1-foot0-~foot1-~foot2-~isnew-small_capq-minute-7'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'~opentop2-~speedup2-foot0-~foot1-~foot2-~isnew-small_capq-minute-7'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0msub_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_keys\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mvalid_keys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mremove_invalid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msub_df\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mvalid_keys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'remove_invalid' is not defined"
     ]
    }
   ],
   "source": [
    "valid_keys = ['~opentop2-jump1-foot0-~foot1-~foot2-~isnew-small_capq-minute-7', '~opentop2-~speedup2-foot0-~foot1-~foot2-~isnew-small_capq-minute-7']\n",
    "sub_df = df[df.index.isin(valid_keys) == True]\n",
    "valid_keys = remove_invalid(sub_df,7)\n",
    "valid_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/ipykernel/__main__.py:14: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "/usr/local/lib/python2.7/site-packages/ipykernel/__main__.py:15: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set(['~opentop2-speedup1-~speedup2-~jump2-foot0-~small_capq-minute-2', '~jump1-jump2-foot1-foot2-~isnew-~small_capq-small_volume-~opentop1-top1-2', '~speedup1-~jump1-speedup2-foot0-foot1-~isnew-~small_capq-2', '~opentop2-~jump2-~foot0-~foot2-isnew-small_volume-2', 'speedup1-~jump1-speedup2-~foot1-~foot2-~isnew-small_capq-small_volume-2', '~opentop2-~speedup1-~speedup2-foot1-~foot2-isnew-minute-2', '~opentop2-speedup1-foot1-foot2-small_capq-small_volume-2', '~opentop2-~speedup1-jump1-jump2-~foot1-small_volume-2'])\n",
      "8 182 2\n",
      "--------\n",
      "set(['jump1-~foot1-~foot2-~isnew-minute-small_volume-yz0-3', '~opentop2-~jump1-~jump2-foot0-~foot2-~isnew-small_capq-minute-3', 'jump1-~foot1-~isnew-small_capq-minute-small_volume-yz0-3', '~speedup1-jump1-~speedup2-~jump2-~foot1-~foot2-minute-~small_volume-~opentop0-top0-~opentop1-top1-3', '~opentop2-~speedup1-jump2-foot0-~foot1-minute-small_volume-~opentop1-top1-3', 'jump2-foot0-~foot1-foot2-~isnew-small_capq-small_volume-3', '~opentop2-~speedup2-jump2-foot0-~foot1-~foot2-small_capq-~small_volume-3', '~opentop2-jump1-jump2-~foot1-foot2-~small_capq-minute-~opentop1-top1-3', '~opentop2-~jump1-~jump2-~foot0-~foot1-~small_capq-minute-3', 'opentop2-~speedup1-jump1-~foot1-~foot2-small_volume-~opentop1-top1-3', '~opentop2-speedup1-foot0-~foot2-~isnew-~small_capq-small_volume-3', 'jump1-speedup2-foot1-~isnew-small_capq-small_volume-~opentop0-top0-~opentop1-top1-3', 'opentop2-jump1-jump2-foot1-minute-~opentop1-top1-3', '~opentop2-jump2-foot0-~foot1-~foot2-~small_capq-small_volume-3', '~speedup1-speedup2-foot0-~foot1-~isnew-small_capq-small_volume-~opentop1-top1-3', '~speedup1-jump1-~jump2-foot0-~foot1-minute-~yz1-top1-opentop1-3', '~opentop2-speedup1-jump1-jump2-~foot1-foot2-~small_capq-~opentop1-top1-3', '~opentop2-~speedup1-foot0-~foot1-~foot2-minute-~small_volume-~yz1-top1-opentop1-3'])\n",
      "18 2396 3\n",
      "--------\n",
      "set(['~opentop2-~speedup1-~foot2-~isnew-~small_capq-minute-small_volume-~opentop1-top1-4', '~opentop2-~speedup1-~jump2-~foot1-~foot2-small_capq-small_volume-~opentop0-top0-4', '~opentop2-jump1-~speedup2-~foot1-minute-small_volume-~yz1-top1-opentop1-4', '~speedup1-jump1-~speedup2-~foot0-~foot2-~isnew-minute-~opentop0-top0-4', '~speedup1-jump1-jump2-~foot1-foot2-small_volume-4', '~opentop2-~speedup1-jump1-~speedup2-~foot1-~foot2-~small_capq-minute-4', '~opentop2-jump2-~foot1-~isnew-small_volume-yz0-4', '~opentop2-~speedup1-jump1-~speedup2-~foot1-minute-small_volume-4', '~opentop2-~speedup1-jump1-foot0-~foot1-minute-small_volume-4', '~opentop2-~jump1-speedup2-small_capq-small_volume-~opentop1-top1-4', '~speedup1-foot0-~foot1-foot2-small_volume-4', '~speedup1-~jump2-~foot1-~isnew-small_capq-minute-small_volume-4', '~opentop2-~foot1-~isnew-small_capq-minute-small_volume-yz0-4', '~opentop2-~speedup2-~foot0-~isnew-small_capq-minute-small_volume-~opentop0-top0-4', '~opentop2-jump1-~speedup2-~isnew-minute-small_volume-~yz1-top1-opentop1-4', '~opentop2-~speedup1-jump1-minute-small_volume-~yz1-top1-opentop1-4', '~opentop2-jump2-foot2-small_capq-small_volume-4', '~opentop2-speedup2-foot2-small_capq-minute-~opentop1-top1-4'])\n",
      "18 4554 4\n",
      "--------\n",
      "set(['jump1-~speedup2-~foot2-~isnew-minute-~yz1-top1-opentop1-5', '~speedup1-~speedup2-~foot0-minute-~small_volume-~opentop1-top1-5', '~jump1-~speedup2-~foot1-~isnew-~opentop1-top1-5', 'speedup1-jump1-~speedup2-~foot0-~foot2-small_volume-~opentop0-top0-5', '~opentop2-~speedup1-~jump1-~foot1-5'])\n",
      "5 379 5\n",
      "--------\n",
      "set(['~opentop2-jump1-small_capq-minute-small_volume-6', '~opentop2-~speedup1-foot0-~foot1-small_capq-minute-6', 'jump1-~speedup2-~foot2-~isnew-small_capq-minute-small_volume-6'])\n",
      "3 582 6\n",
      "--------\n",
      "set(['~foot2-small_volume-yz0-7'])\n",
      "1 29 7\n",
      "--------\n"
     ]
    }
   ],
   "source": [
    "for i in range(2,8):\n",
    "    sub_df = df[df[\"recent\"] == str(i)]\n",
    "    valid_keys = remove_invalid(sub_df,i)\n",
    "    print valid_keys\n",
    "    print len(valid_keys),len(sub_df),i\n",
    "    print \"--------\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def remove_invalid(output_df,day):\n",
    "    keys = output_df.index\n",
    "    valid_keys = set(keys)\n",
    "    for keyA in keys:\n",
    "        if not keyA in valid_keys:\n",
    "            continue\n",
    "        filterA = getFilter(keyA)\n",
    "        filter_recent = recent(day,data_df)\n",
    "        for keyB in keys:\n",
    "            if not keyB in valid_keys or keyA == keyB:\n",
    "                continue\n",
    "#             print keyA + \"--\" + keyB\n",
    "            filterB = getFilter(keyB)\n",
    "            sub_dfA = data_df[filter_recent & filterA & ~filterB & ~yz2]\n",
    "            sub_dfB = data_df[filter_recent & filterB & ~filterA & ~yz2]\n",
    "            to_remove,to_break = shouldRemove(keyA,keyB,sub_dfA,sub_dfB,output_df)\n",
    "            if to_remove is not None:\n",
    "                valid_keys.remove(to_remove)\n",
    "            if to_break:\n",
    "                break            \n",
    "    return valid_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def shouldRemove(keyA,keyB,sub_dfA,sub_dfB,output_df):\n",
    "    if is_valid(sub_dfA) and not is_valid(sub_dfB):\n",
    "        return keyB,False\n",
    "    elif not is_valid(sub_dfA) and is_valid(sub_dfB):\n",
    "        return keyA,True\n",
    "    elif not is_valid(sub_dfA) and not is_valid(sub_dfB):\n",
    "        if output_df[\"count\"][keyA] > output_df[\"count\"][keyB]:\n",
    "            return keyB,False\n",
    "        elif output_df[\"count\"][keyA] < output_df[\"count\"][keyB]:\n",
    "            return keyA,True\n",
    "        else:\n",
    "            if keyA.split(\"-\") < keyB.split(\"-\"):\n",
    "                return keyB,False\n",
    "            elif keyA.split(\"-\") > keyB.split(\"-\"):\n",
    "                return keyA,True\n",
    "            else:\n",
    "                if output_df[\"win_ratio\"][keyA] >= output_df[\"win_ratio\"][keyB]:\n",
    "                    return keyB,False\n",
    "                else:\n",
    "                    return keyA,True\n",
    "    else:\n",
    "        return None,False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def is_valid(sub_df):\n",
    "    sub_df = sub_df.sort_values(\"minute\")\n",
    "    sub_df = sub_df.groupby(\"date\").first()\n",
    "    count = len(sub_df)\n",
    "    mean = sub_df.change.mean()\n",
    "    if mean < 1.015 or count < 2:\n",
    "        return False \n",
    "    win_df = sub_df[sub_df[\"change\"] > 1.0]\n",
    "    win_ratio = float(len(win_df))/len(sub_df)\n",
    "    if count < 10:\n",
    "        return mean > 1.025 and win_ratio > 0.75\n",
    "    else:\n",
    "        return mean > 1.02 and win_ratio > 0.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/ipykernel/__main__.py:7: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n"
     ]
    }
   ],
   "source": [
    "win_ratio_se = pd.Series()\n",
    "mean_se = pd.Series()\n",
    "for filters in filterss:\n",
    "    each_filters = []\n",
    "    for filter_key in filters:\n",
    "        filter_a = getFilter(filter_key,True)\n",
    "        sub_df = data_df[filter_a]\n",
    "        sub_df = sub_df.sort_values(\"minute\")\n",
    "        sub_df = sub_df.groupby(\"date\").first()\n",
    "        count = len(sub_df)\n",
    "        mean = sub_df.change.mean()\n",
    "        win_df = sub_df[sub_df[\"change\"] > 1.0]\n",
    "        win_ratio = float(len(win_df))/len(sub_df)\n",
    "        win_ratio_se[filter_key] = win_ratio\n",
    "        mean_se[filter_key] = mean\n",
    "df = pd.DataFrame()\n",
    "df[\"win_ratio\"] = win_ratio_se\n",
    "df[\"mean\"] = mean_se\n",
    "df.to_csv(\"output4.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
